<?php
    require_once ('tokenizer.php');

    /******* TOKENIZER EXAMPLES ********/

    // Simple example
    /*
        [0]=> "Hello"
        [1]=> " "
        [2]=> "world!"
    */
    $tokenizer = tokenizer::registerDefaultTokenizer();
    var_dump ($tokenizer->tokenize ('Hello world!'));

    // Example with markers
    /*
        [0]=> "Please"
        [1]=> " "
        [2]=> "not"
        [3]=> " "
        [4]=> ""cut quotation points"!"
    */
    var_dump ($tokenizer->tokenize ('Please not "cut quotation points"!'));

    // Example with escaping char
    /*
        [0]=> "Mark"
        [1]=> " "
        [2]=> "said:"
        [3]=> " "
        [4]=> "'where is Janet\'s jacket'?"
    */
    var_dump ($tokenizer->tokenize ("Mark said: 'where is Janet\'s jacket'?"));
?>